﻿#from numpy import array
from numpy import *
from NewsMining.db.dbConnect import getDBConnection
from mynewsweb.model import *
from math import sqrt
from NewsMining.crawler.preprocessText import *
from sskFind import *
import orange
import orngSVM
from orngSVM import KernelWrapper
import numpy as np
#from cache import cached_kernel

#==== ==== ==== ====
# Settings
#==== ==== ==== ====
k = 7
decay = 0.5
order = 1
smitran = 2

sample_length = 10

print "Settings: k: %s, decay: %s, order: %s, smitran: %s" % (k, decay, order, smitran)

#==== ==== ==== ====
# CACHE
#==== ==== ==== ====
def cached_kernel(kernel, maxSize=10000000):
    cache = {}
    def kernel2(x, y, data, k=0, decay=0, order=1, smitran=1):
        if cache.has_key((x,y)):
            return cache[(x,y)]
        if cache.has_key((y,x)):
            return cache[(y,x)]
        #print "Run kernel"
        value = kernel(x, y, data, k, decay, order, smitran)
        if len(cache) == maxSize:
            cache.popitem()
        cache[(x,y)] = value
        return value
    return kernel2

#==== ==== ==== ====
# SMITRAN v 2.0
#==== ==== ==== ====
def smitran_kernel2(s1, s2, data, k, decay, order, smitran):
    s1d = data[s1]
    s2d = data[s2]
    return sum([s1d[i] * s2d[i] for i in s1d])

    #return sum(data[s1].values()) * sum(data[s2].values())

#==== ==== ==== ====
# ORANGE SVM
#==== ==== ==== ====
print "Create Orange examples"

# Data in orange format
domainDef = [orange.EnumVariable(name="class_id", values=["1", "-1"])] #, orange.StringVariable("news_id")
domain = orange.Domain(domainDef)

news_id_var = orange.StringVariable("news_id")
id = orange.newmetaid()
domain.addmeta(id, news_id_var)

#==== ==== ==== ====
# DATA
#==== ==== ==== ====
title_unicode_trans = TitleUnicodeTranslate()

# ==== TEST ====
test = {0: u"Dusan Smitran",
        10: u"Dusan Smitran",
        1:  u"Dusanov Smitranovi",
        2:  u"Grega Kovacic",
        3:  u"Lekarni Ljubljana pol leta prodajali brez registracije, bodo na ministrstvu ukrepali oziroma bo to storila agencija za zdravila, je zatrdil minis1",
        4:  u"Lekarne Ljubljana so s polic umaknile sporna zdravila. (Foto: Anže Petkovšek) Kot se je izkazalo kasneje, je bilo sedem izdelkov, ki so jih v Lekarni Ljubljana prodajali kot ",
        5:  u"vana o vsem skupaj ne govori. (Foto: Borut Cvetko, Mediaspeed) Estradnica Ivana &Scaron;undov Hojan , ki živi v Zagrebu, se je v medijih pohvalila, da je postala zastopnica za popularne uggice. Direktor podjetja Logos Trend Miha Hrovat trdi, da Ivana pro"}

test_data = {}

for t in test:
    text = removeStopWords(simpleNormalize(test[t], title_unicode_trans)) #removeStopWords(preprocess(test[t]))
    ngrams = array([text[i:i + k] for i in range(0, len(text) - k + 1, 2)])
    charIndexes = dict([(i, allindex(text, i)) for i in set(text)])
    if len(text) > 10:
        test_data[t] = [ngrams, charIndexes]

# ==== REAL ====
db = getDBConnection()
#news = db.query(News).filter(News.rank != None)[:20]
all_news_top = db.query(News).filter(News.content != '').filter(News.class_id == 1)[:sample_length]
all_news_low = db.query(News).filter(News.content != '').filter(News.class_id == -1)[:sample_length]

news = all_news_top + all_news_low

news_data = {}
examples = []

# GENERATE Ngrams FOR News + ALL GRAMS = S'
all_ngrams = {}

for n in news:
    text = removeStopWords(simpleNormalize(n.content, title_unicode_trans))
    ngrams = array([text[i:i + k] for i in range(0, len(text) - k + 1, 1)])

    for ngram in ngrams:
        if all_ngrams.has_key(ngram):
            all_ngrams[ngram] += 1
        else:
            all_ngrams[ngram] = 1

    charIndexes = dict([(i, allindex(text, i)) for i in set(text)])
    if len(text) > 300:
        news_data[n.news_id] = charIndexes

        # Orange
        o = [str(n.class_id)]
        ex = orange.Example(domain, o)
        ex['news_id'] = str(n.news_id)
        #print o
        examples.append(ex)

data = orange.ExampleTable(examples)

# TOP 1000 freqvent ngrams
print "TOP 1000 freqvent ngrams"
sgrams = all_ngrams.items()
sgrams.sort(lambda x,y: -cmp(x[1], y[1]))
sgrams = sgrams[:1000]
d = dict([(i[0], 0) for i in sgrams])

# Calculate News with Top 1000 ngrams
print "Calculate News with Top 1000 ngrams"
news_produkt = {}

for n in news_data:
    print n
    charIndexes = news_data[n]
    news_sgrams = d.copy()

    for ngram in news_sgrams:
        stringIndexes = [charIndexes.get(i, []) for i in ngram]
        sskFind = sskFindOut(k, order, stringIndexes)
        sskFind(-1, 0)
        odklon = sum([decay ** (odklon ** smitran) for odklon in sskFind(-99)])
        news_sgrams[ngram] = odklon

    news_produkt[n] = news_sgrams


#sdfdsf
#==== ==== ==== ====
# CALL
#==== ==== ==== ====
print "Run kernel"
kernel = cached_kernel(smitran_kernel2)

#dfsfsdf
n1 = 249 #66, 249#
# ONE vs ALL smitran 2
##for n2 in news_data.keys()[:]:
##    if n1 != n2: # and n2 == 183: # and n2 in [142, 147, 392, 84]: #142, 147, 392, 84
##        a = kernel(n1, n2, news_data, k, decay, order, smitran)
##        b = kernel(n1, n1, news_data, k, decay, order, smitran)
##        c = kernel(n2, n2, news_data, k, decay, order, smitran)
##        d = a / sqrt(b * c)
##        print "*" * 10
##        print n1, n2
##        print a, b, c
##        print d
##        if d > 0:
##            r = KernelResults(n1, n2, d, 2, k, decay, order, smitran)
##            db.add(r)
##
##    db.commit()


### ALL vs ALL smitran2
##i = 0
##for n1 in news_data.keys():
##    print n1
##    for n2 in news_data.keys():
##        if n1 != n2:
##            a = kernel(n1, n2, news_produkt, k, decay, order, smitran)
##            b = kernel(n1, n1, news_produkt, k, decay, order, smitran)
##            c = kernel(n2, n2, news_produkt, k, decay, order, smitran)
##            d = a / sqrt(b * c)
##            if d > 0:
##                r = KernelResults(n1, n2, d, 2, k, decay, order, smitran)
##                db.add(r)
##
##    i += 1
##    if i % 100 == 0:
##        print "Commit"
##        db.commit()
##
##db.commit()

#==== ==== ==== ====
# SVM - kernel
#==== ==== ==== ====
print "Run SVM!!!"

def intermediate_kernel(n1, n2):
    return kernel(n1, n2, news_produkt, k, decay, order, smitran) / sqrt(kernel(n1, n1, news_produkt, k, decay, order, smitran) * kernel(n2, n2, news_produkt, k, decay, order, smitran))

def example_kernel(example1, example2):
    return intermediate_kernel(int(example1['news_id'].value), int(example2['news_id'].value))

# Build SVM
l1=orngSVM.SVMLearner()
l1.kernelFunc=KernelWrapper(example_kernel) #orngSVM. #Linear
l1.kernel_type=orange.SVMLearner.Custom
l1.probability=True
c1=l1(data)
l1.name="SVM - Linear"

l2=orngSVM.SVMLearner()
l2.kernelFunc=orngSVM.RBFKernelWrapper(example_kernel, gamma=0.5)
l2.kernel_type=orange.SVMLearner.Custom
l2.probability=True
c2=l2(data)
l2.name="SVM - RBF"

l3=orngSVM.SVMLearner()
l3.kernelFunc=orngSVM.PolyKernelWrapper(example_kernel, degree=3.0)
l3.kernel_type=orange.SVMLearner.Custom
l3.probability=True
c3=l3(data)
l3.name="SVM - Poly"


# RUN Tests, CA
print "... tests (crossValidation)"
import orngTest, orngStat
tests=orngTest.crossValidation([l1, l2, l3], data, folds=5)
[ca1, ca2, ca3]=orngStat.CA(tests)

print "SVM: Smitran-Ssk"
print "sample_length:", sample_length
print "Settings: k: %s, decay: %s, order: %s, smitran: %s" % (k, decay, order, smitran)
print l1.name, "CA:", ca1
print l2.name, "CA:", ca2
print l3.name, "CA:", ca3